{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BasicConv2d(nn.Module):\n",
    "\n",
    "    def __init__(self, input_channels, output_channels, kernel_size, activation = 'relu', **kwargs):\n",
    "        super().__init__()\n",
    "        self.conv = nn.Conv2d(input_channels, output_channels, kernel_size, **kwargs)\n",
    "        self.bn = nn.BatchNorm2d(output_channels)\n",
    "        \n",
    "        if (activation == 'relu'):\n",
    "            self.relu = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if (activation == 'swish'):\n",
    "            self.relu = swish()\n",
    "            \n",
    "        if (activation == 'mish'):\n",
    "            self.relu = mish()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.conv(x)\n",
    "        x = self.bn(x)\n",
    "        x = self.relu(x)\n",
    "        return x\n",
    "\n",
    "class ChannelShuffle(nn.Module):\n",
    "\n",
    "    def __init__(self, groups):\n",
    "        super().__init__()\n",
    "        self.groups = groups\n",
    "    \n",
    "    def forward(self, x):\n",
    "        batchsize, channels, height, width = x.data.size()\n",
    "        channels_per_group = int(channels / self.groups)\n",
    "\n",
    "        #\"\"\"suppose a convolutional layer with g groups whose output has\n",
    "        #g x n channels; we first reshape the output channel dimension\n",
    "        #into (g, n)\"\"\"\n",
    "        x = x.view(batchsize, self.groups, channels_per_group, height, width)\n",
    "\n",
    "        #\"\"\"transposing and then flattening it back as the input of next layer.\"\"\"\n",
    "        x = x.transpose(1, 2).contiguous()\n",
    "        x = x.view(batchsize, -1, height, width)\n",
    "\n",
    "        return x\n",
    "\n",
    "class DepthwiseConv2d(nn.Module):\n",
    "\n",
    "    def __init__(self, input_channels, output_channels, kernel_size, **kwargs):\n",
    "        super().__init__()\n",
    "        self.depthwise = nn.Sequential(\n",
    "            nn.Conv2d(input_channels, output_channels, kernel_size, **kwargs),\n",
    "            nn.BatchNorm2d(output_channels)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.depthwise(x)\n",
    "\n",
    "class PointwiseConv2d(nn.Module):\n",
    "    def __init__(self, input_channels, output_channels, **kwargs):\n",
    "        super().__init__()\n",
    "        self.pointwise = nn.Sequential(\n",
    "            nn.Conv2d(input_channels, output_channels, 1, **kwargs),\n",
    "            nn.BatchNorm2d(output_channels)\n",
    "        )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return self.pointwise(x)\n",
    "\n",
    "class ShuffleNetUnit(nn.Module):\n",
    "\n",
    "    def __init__(self, input_channels, output_channels, stage, stride, groups, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        \n",
    "        if (activation == 'relu'):\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if (activation == 'swish'):\n",
    "            f_activation = swish()\n",
    "            \n",
    "        if (activation == 'mish'):\n",
    "            f_activation = mish()\n",
    "\n",
    "        #\"\"\"Similar to [9], we set the number of bottleneck channels to 1/4 \n",
    "        #of the output channels for each ShuffleNet unit.\"\"\"\n",
    "        self.bottlneck = nn.Sequential(\n",
    "            PointwiseConv2d(\n",
    "                input_channels, \n",
    "                int(output_channels / 4), \n",
    "                groups=groups\n",
    "            ),\n",
    "            f_activation\n",
    "        )\n",
    "\n",
    "        #\"\"\"Note that for Stage 2, we do not apply group convolution on the first pointwise \n",
    "        #layer because the number of input channels is relatively small.\"\"\"\n",
    "        if stage == 2:\n",
    "            self.bottlneck = nn.Sequential(\n",
    "                PointwiseConv2d(\n",
    "                    input_channels, \n",
    "                    int(output_channels / 4),\n",
    "                    groups=groups\n",
    "                ),\n",
    "                f_activation\n",
    "            )\n",
    "        \n",
    "        self.channel_shuffle = ChannelShuffle(groups)\n",
    "\n",
    "        self.depthwise = DepthwiseConv2d(\n",
    "            int(output_channels / 4), \n",
    "            int(output_channels / 4), \n",
    "            3, \n",
    "            groups=int(output_channels / 4), \n",
    "            stride=stride,\n",
    "            padding=1\n",
    "        )\n",
    "\n",
    "        self.expand = PointwiseConv2d(\n",
    "            int(output_channels / 4),\n",
    "            output_channels,\n",
    "            groups=groups\n",
    "        )\n",
    "\n",
    "        self.relu = f_activation\n",
    "        self.fusion = self._add\n",
    "        self.shortcut = nn.Sequential()\n",
    "\n",
    "        #\"\"\"As for the case where ShuffleNet is applied with stride, \n",
    "        #we simply make two modifications (see Fig 2 (c)): \n",
    "        #(i) add a 3 × 3 average pooling on the shortcut path; \n",
    "        #(ii) replace the element-wise addition with channel concatenation, \n",
    "        #which makes it easy to enlarge channel dimension with little extra \n",
    "        #computation cost.\n",
    "        if stride != 1 or input_channels != output_channels:\n",
    "            self.shortcut = nn.AvgPool2d(3, stride=2, padding=1)\n",
    "\n",
    "            self.expand = PointwiseConv2d(\n",
    "                int(output_channels / 4),\n",
    "                output_channels - input_channels,\n",
    "                groups=groups\n",
    "            )\n",
    "\n",
    "            self.fusion = self._cat\n",
    "    \n",
    "    def _add(self, x, y):\n",
    "        return torch.add(x, y)\n",
    "    \n",
    "    def _cat(self, x, y):\n",
    "        return torch.cat([x, y], dim=1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        shortcut = self.shortcut(x)\n",
    "\n",
    "        shuffled = self.bottlneck(x)\n",
    "        shuffled = self.channel_shuffle(shuffled)\n",
    "        shuffled = self.depthwise(shuffled)\n",
    "        shuffled = self.expand(shuffled)\n",
    "\n",
    "        output = self.fusion(shortcut, shuffled)\n",
    "        output = self.relu(output)\n",
    "\n",
    "        return output\n",
    "\n",
    "class ShuffleNet(nn.Module):\n",
    "\n",
    "    def __init__(self, num_blocks, num_classes=10, groups=3, activation = 'relu'):\n",
    "        super().__init__()\n",
    "\n",
    "        if groups == 1:\n",
    "            out_channels = [24, 144, 288, 567]\n",
    "        elif groups == 2:\n",
    "            out_channels = [24, 200, 400, 800]\n",
    "        elif groups == 3:\n",
    "            out_channels = [24, 240, 480, 960]\n",
    "        elif groups == 4:\n",
    "            out_channels = [24, 272, 544, 1088]\n",
    "        elif groups == 8:\n",
    "            out_channels = [24, 384, 768, 1536]\n",
    "\n",
    "        self.conv1 = BasicConv2d(3, out_channels[0], 3, padding=1, stride=1, activation = activation)\n",
    "        self.input_channels = out_channels[0]\n",
    "\n",
    "        self.stage2 = self._make_stage(\n",
    "            ShuffleNetUnit, \n",
    "            num_blocks[0], \n",
    "            out_channels[1], \n",
    "            stride=2, \n",
    "            stage=2,\n",
    "            groups=groups,\n",
    "            activation = activation\n",
    "        )\n",
    "\n",
    "        self.stage3 = self._make_stage(\n",
    "            ShuffleNetUnit, \n",
    "            num_blocks[1], \n",
    "            out_channels[2], \n",
    "            stride=2,\n",
    "            stage=3, \n",
    "            groups=groups,\n",
    "            activation = activation\n",
    "        )\n",
    "\n",
    "        self.stage4 = self._make_stage(\n",
    "            ShuffleNetUnit,\n",
    "            num_blocks[2],\n",
    "            out_channels[3],\n",
    "            stride=2,\n",
    "            stage=4,\n",
    "            groups=groups,\n",
    "            activation = activation\n",
    "        )\n",
    "\n",
    "        self.avg = nn.AdaptiveAvgPool2d((1, 1))\n",
    "        self.fc = nn.Linear(out_channels[3], num_classes)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.stage2(x)\n",
    "        x = self.stage3(x)\n",
    "        x = self.stage4(x)\n",
    "        x = self.avg(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.fc(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "    def _make_stage(self, block, num_blocks, output_channels, stride, stage, groups, activation = 'relu'):\n",
    "        \"\"\"make shufflenet stage \n",
    "\n",
    "        Args:\n",
    "            block: block type, shuffle unit\n",
    "            out_channels: output depth channel number of this stage\n",
    "            num_blocks: how many blocks per stage\n",
    "            stride: the stride of the first block of this stage\n",
    "            stage: stage index\n",
    "            groups: group number of group convolution \n",
    "        Return:\n",
    "            return a shuffle net stage\n",
    "        \"\"\"\n",
    "        strides = [stride] + [1] * (num_blocks - 1)\n",
    "\n",
    "        stage = []\n",
    "\n",
    "        for stride in strides:\n",
    "            stage.append(\n",
    "                block(\n",
    "                    self.input_channels, \n",
    "                    output_channels, \n",
    "                    stride=stride, \n",
    "                    stage=stage, \n",
    "                    groups=groups,\n",
    "                    activation = activation\n",
    "                )\n",
    "            )\n",
    "            self.input_channels = output_channels\n",
    "\n",
    "        return nn.Sequential(*stage)\n",
    "\n",
    "def shufflenet(activation = 'relu'):\n",
    "    return ShuffleNet([4, 8, 4], activation = activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:06, 26673113.11it/s]                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = shufflenet(activation = 'mish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 42.1768.. Average time per step: 0.1079.. Train loss: 1.3796.. Train accuracy: 0.4980.. Top-3 train accuracy: 0.8115.. Test loss: 1.0749.. Test accuracy: 0.6163.. Top-3 test accuracy: 0.8839\n",
      "Epoch 2/100.. Time per epoch: 41.4615.. Average time per step: 0.1060.. Train loss: 1.0233.. Train accuracy: 0.6381.. Top-3 train accuracy: 0.8931.. Test loss: 0.8531.. Test accuracy: 0.6997.. Top-3 test accuracy: 0.9252\n",
      "Epoch 3/100.. Time per epoch: 41.4102.. Average time per step: 0.1059.. Train loss: 0.8647.. Train accuracy: 0.6938.. Top-3 train accuracy: 0.9202.. Test loss: 0.7527.. Test accuracy: 0.7397.. Top-3 test accuracy: 0.9383\n",
      "Epoch 4/100.. Time per epoch: 41.3561.. Average time per step: 0.1058.. Train loss: 0.7762.. Train accuracy: 0.7282.. Top-3 train accuracy: 0.9334.. Test loss: 0.7522.. Test accuracy: 0.7427.. Top-3 test accuracy: 0.9413\n",
      "Epoch 5/100.. Time per epoch: 41.4109.. Average time per step: 0.1059.. Train loss: 0.7028.. Train accuracy: 0.7553.. Top-3 train accuracy: 0.9426.. Test loss: 0.6690.. Test accuracy: 0.7712.. Top-3 test accuracy: 0.9484\n",
      "Epoch 6/100.. Time per epoch: 41.1837.. Average time per step: 0.1053.. Train loss: 0.6518.. Train accuracy: 0.7720.. Top-3 train accuracy: 0.9503.. Test loss: 0.6549.. Test accuracy: 0.7679.. Top-3 test accuracy: 0.9578\n",
      "Epoch 7/100.. Time per epoch: 41.3231.. Average time per step: 0.1057.. Train loss: 0.6118.. Train accuracy: 0.7876.. Top-3 train accuracy: 0.9549.. Test loss: 0.5681.. Test accuracy: 0.8060.. Top-3 test accuracy: 0.9603\n",
      "Epoch 8/100.. Time per epoch: 41.5606.. Average time per step: 0.1063.. Train loss: 0.5706.. Train accuracy: 0.7994.. Top-3 train accuracy: 0.9599.. Test loss: 0.5983.. Test accuracy: 0.7998.. Top-3 test accuracy: 0.9589\n",
      "Epoch 9/100.. Time per epoch: 41.2895.. Average time per step: 0.1056.. Train loss: 0.5373.. Train accuracy: 0.8139.. Top-3 train accuracy: 0.9640.. Test loss: 0.5637.. Test accuracy: 0.8046.. Top-3 test accuracy: 0.9644\n",
      "Epoch 10/100.. Time per epoch: 41.4478.. Average time per step: 0.1060.. Train loss: 0.5127.. Train accuracy: 0.8202.. Top-3 train accuracy: 0.9664.. Test loss: 0.5247.. Test accuracy: 0.8187.. Top-3 test accuracy: 0.9668\n",
      "Epoch 11/100.. Time per epoch: 41.3137.. Average time per step: 0.1057.. Train loss: 0.4847.. Train accuracy: 0.8297.. Top-3 train accuracy: 0.9697.. Test loss: 0.5171.. Test accuracy: 0.8283.. Top-3 test accuracy: 0.9670\n",
      "Epoch 12/100.. Time per epoch: 41.4069.. Average time per step: 0.1059.. Train loss: 0.4663.. Train accuracy: 0.8374.. Top-3 train accuracy: 0.9714.. Test loss: 0.5069.. Test accuracy: 0.8300.. Top-3 test accuracy: 0.9685\n",
      "Epoch 13/100.. Time per epoch: 41.4968.. Average time per step: 0.1061.. Train loss: 0.4431.. Train accuracy: 0.8450.. Top-3 train accuracy: 0.9740.. Test loss: 0.4934.. Test accuracy: 0.8354.. Top-3 test accuracy: 0.9683\n",
      "Epoch 14/100.. Time per epoch: 41.5604.. Average time per step: 0.1063.. Train loss: 0.4292.. Train accuracy: 0.8497.. Top-3 train accuracy: 0.9759.. Test loss: 0.5215.. Test accuracy: 0.8296.. Top-3 test accuracy: 0.9664\n",
      "Epoch 15/100.. Time per epoch: 41.4341.. Average time per step: 0.1060.. Train loss: 0.4129.. Train accuracy: 0.8548.. Top-3 train accuracy: 0.9774.. Test loss: 0.4815.. Test accuracy: 0.8381.. Top-3 test accuracy: 0.9691\n",
      "Epoch 16/100.. Time per epoch: 41.4337.. Average time per step: 0.1060.. Train loss: 0.3894.. Train accuracy: 0.8640.. Top-3 train accuracy: 0.9792.. Test loss: 0.4841.. Test accuracy: 0.8402.. Top-3 test accuracy: 0.9693\n",
      "Epoch 17/100.. Time per epoch: 41.3644.. Average time per step: 0.1058.. Train loss: 0.3737.. Train accuracy: 0.8688.. Top-3 train accuracy: 0.9807.. Test loss: 0.4690.. Test accuracy: 0.8469.. Top-3 test accuracy: 0.9721\n",
      "Epoch 18/100.. Time per epoch: 41.3763.. Average time per step: 0.1058.. Train loss: 0.3654.. Train accuracy: 0.8728.. Top-3 train accuracy: 0.9808.. Test loss: 0.4723.. Test accuracy: 0.8440.. Top-3 test accuracy: 0.9707\n",
      "Epoch 19/100.. Time per epoch: 41.3392.. Average time per step: 0.1057.. Train loss: 0.3466.. Train accuracy: 0.8789.. Top-3 train accuracy: 0.9827.. Test loss: 0.4737.. Test accuracy: 0.8505.. Top-3 test accuracy: 0.9725\n",
      "Epoch 20/100.. Time per epoch: 41.1556.. Average time per step: 0.1053.. Train loss: 0.3404.. Train accuracy: 0.8810.. Top-3 train accuracy: 0.9834.. Test loss: 0.4871.. Test accuracy: 0.8462.. Top-3 test accuracy: 0.9714\n",
      "Epoch 21/100.. Time per epoch: 41.3734.. Average time per step: 0.1058.. Train loss: 0.3250.. Train accuracy: 0.8853.. Top-3 train accuracy: 0.9850.. Test loss: 0.4684.. Test accuracy: 0.8511.. Top-3 test accuracy: 0.9729\n",
      "Epoch 22/100.. Time per epoch: 41.0852.. Average time per step: 0.1051.. Train loss: 0.3106.. Train accuracy: 0.8894.. Top-3 train accuracy: 0.9858.. Test loss: 0.4670.. Test accuracy: 0.8527.. Top-3 test accuracy: 0.9730\n",
      "Epoch 23/100.. Time per epoch: 41.2893.. Average time per step: 0.1056.. Train loss: 0.3052.. Train accuracy: 0.8922.. Top-3 train accuracy: 0.9859.. Test loss: 0.4558.. Test accuracy: 0.8570.. Top-3 test accuracy: 0.9750\n",
      "Epoch 24/100.. Time per epoch: 41.2532.. Average time per step: 0.1055.. Train loss: 0.2905.. Train accuracy: 0.8972.. Top-3 train accuracy: 0.9884.. Test loss: 0.4593.. Test accuracy: 0.8528.. Top-3 test accuracy: 0.9751\n",
      "Epoch 25/100.. Time per epoch: 41.3672.. Average time per step: 0.1058.. Train loss: 0.2821.. Train accuracy: 0.9003.. Top-3 train accuracy: 0.9882.. Test loss: 0.4586.. Test accuracy: 0.8541.. Top-3 test accuracy: 0.9745\n",
      "Epoch 26/100.. Time per epoch: 41.3558.. Average time per step: 0.1058.. Train loss: 0.2777.. Train accuracy: 0.9021.. Top-3 train accuracy: 0.9879.. Test loss: 0.4481.. Test accuracy: 0.8574.. Top-3 test accuracy: 0.9755\n",
      "Epoch 27/100.. Time per epoch: 41.3178.. Average time per step: 0.1057.. Train loss: 0.2642.. Train accuracy: 0.9074.. Top-3 train accuracy: 0.9904.. Test loss: 0.4622.. Test accuracy: 0.8609.. Top-3 test accuracy: 0.9763\n",
      "Epoch 28/100.. Time per epoch: 41.5516.. Average time per step: 0.1063.. Train loss: 0.2571.. Train accuracy: 0.9097.. Top-3 train accuracy: 0.9899.. Test loss: 0.4691.. Test accuracy: 0.8527.. Top-3 test accuracy: 0.9722\n",
      "Epoch 29/100.. Time per epoch: 41.4139.. Average time per step: 0.1059.. Train loss: 0.2466.. Train accuracy: 0.9121.. Top-3 train accuracy: 0.9909.. Test loss: 0.4697.. Test accuracy: 0.8594.. Top-3 test accuracy: 0.9740\n",
      "Epoch 30/100.. Time per epoch: 41.3164.. Average time per step: 0.1057.. Train loss: 0.2432.. Train accuracy: 0.9152.. Top-3 train accuracy: 0.9914.. Test loss: 0.4631.. Test accuracy: 0.8628.. Top-3 test accuracy: 0.9734\n",
      "Epoch 31/100.. Time per epoch: 41.4807.. Average time per step: 0.1061.. Train loss: 0.2321.. Train accuracy: 0.9190.. Top-3 train accuracy: 0.9917.. Test loss: 0.4774.. Test accuracy: 0.8577.. Top-3 test accuracy: 0.9735\n",
      "Epoch 32/100.. Time per epoch: 41.4874.. Average time per step: 0.1061.. Train loss: 0.2271.. Train accuracy: 0.9201.. Top-3 train accuracy: 0.9925.. Test loss: 0.4541.. Test accuracy: 0.8639.. Top-3 test accuracy: 0.9747\n",
      "Epoch 33/100.. Time per epoch: 41.3594.. Average time per step: 0.1058.. Train loss: 0.2270.. Train accuracy: 0.9203.. Top-3 train accuracy: 0.9923.. Test loss: 0.4633.. Test accuracy: 0.8591.. Top-3 test accuracy: 0.9723\n",
      "Epoch 34/100.. Time per epoch: 41.3114.. Average time per step: 0.1057.. Train loss: 0.2166.. Train accuracy: 0.9232.. Top-3 train accuracy: 0.9925.. Test loss: 0.4852.. Test accuracy: 0.8568.. Top-3 test accuracy: 0.9740\n",
      "Epoch 35/100.. Time per epoch: 41.3742.. Average time per step: 0.1058.. Train loss: 0.2072.. Train accuracy: 0.9264.. Top-3 train accuracy: 0.9933.. Test loss: 0.4674.. Test accuracy: 0.8617.. Top-3 test accuracy: 0.9746\n",
      "Epoch 36/100.. Time per epoch: 40.9898.. Average time per step: 0.1048.. Train loss: 0.1995.. Train accuracy: 0.9295.. Top-3 train accuracy: 0.9934.. Test loss: 0.4830.. Test accuracy: 0.8622.. Top-3 test accuracy: 0.9744\n",
      "Epoch 37/100.. Time per epoch: 41.1710.. Average time per step: 0.1053.. Train loss: 0.2001.. Train accuracy: 0.9300.. Top-3 train accuracy: 0.9940.. Test loss: 0.4687.. Test accuracy: 0.8625.. Top-3 test accuracy: 0.9777\n",
      "Epoch 38/100.. Time per epoch: 41.1290.. Average time per step: 0.1052.. Train loss: 0.1924.. Train accuracy: 0.9316.. Top-3 train accuracy: 0.9938.. Test loss: 0.4923.. Test accuracy: 0.8615.. Top-3 test accuracy: 0.9750\n",
      "Epoch 39/100.. Time per epoch: 41.3215.. Average time per step: 0.1057.. Train loss: 0.1877.. Train accuracy: 0.9336.. Top-3 train accuracy: 0.9948.. Test loss: 0.4942.. Test accuracy: 0.8616.. Top-3 test accuracy: 0.9754\n",
      "Epoch 40/100.. Time per epoch: 41.0229.. Average time per step: 0.1049.. Train loss: 0.1830.. Train accuracy: 0.9350.. Top-3 train accuracy: 0.9947.. Test loss: 0.4966.. Test accuracy: 0.8583.. Top-3 test accuracy: 0.9749\n",
      "Epoch 41/100.. Time per epoch: 41.1128.. Average time per step: 0.1051.. Train loss: 0.1781.. Train accuracy: 0.9368.. Top-3 train accuracy: 0.9953.. Test loss: 0.4962.. Test accuracy: 0.8594.. Top-3 test accuracy: 0.9754\n",
      "Epoch 42/100.. Time per epoch: 41.1284.. Average time per step: 0.1052.. Train loss: 0.1774.. Train accuracy: 0.9372.. Top-3 train accuracy: 0.9957.. Test loss: 0.4740.. Test accuracy: 0.8625.. Top-3 test accuracy: 0.9757\n",
      "Epoch 43/100.. Time per epoch: 41.3148.. Average time per step: 0.1057.. Train loss: 0.1697.. Train accuracy: 0.9402.. Top-3 train accuracy: 0.9954.. Test loss: 0.4769.. Test accuracy: 0.8669.. Top-3 test accuracy: 0.9757\n",
      "Epoch 44/100.. Time per epoch: 41.2539.. Average time per step: 0.1055.. Train loss: 0.1656.. Train accuracy: 0.9415.. Top-3 train accuracy: 0.9954.. Test loss: 0.4920.. Test accuracy: 0.8658.. Top-3 test accuracy: 0.9766\n",
      "Epoch 45/100.. Time per epoch: 41.1564.. Average time per step: 0.1053.. Train loss: 0.1585.. Train accuracy: 0.9450.. Top-3 train accuracy: 0.9960.. Test loss: 0.5072.. Test accuracy: 0.8623.. Top-3 test accuracy: 0.9751\n",
      "Epoch 46/100.. Time per epoch: 41.1213.. Average time per step: 0.1052.. Train loss: 0.1586.. Train accuracy: 0.9440.. Top-3 train accuracy: 0.9960.. Test loss: 0.4918.. Test accuracy: 0.8642.. Top-3 test accuracy: 0.9756\n",
      "Epoch 47/100.. Time per epoch: 40.9989.. Average time per step: 0.1049.. Train loss: 0.1606.. Train accuracy: 0.9431.. Top-3 train accuracy: 0.9957.. Test loss: 0.4752.. Test accuracy: 0.8654.. Top-3 test accuracy: 0.9780\n",
      "Epoch 48/100.. Time per epoch: 41.1174.. Average time per step: 0.1052.. Train loss: 0.1484.. Train accuracy: 0.9479.. Top-3 train accuracy: 0.9964.. Test loss: 0.4864.. Test accuracy: 0.8706.. Top-3 test accuracy: 0.9771\n",
      "Epoch 49/100.. Time per epoch: 41.1694.. Average time per step: 0.1053.. Train loss: 0.1512.. Train accuracy: 0.9469.. Top-3 train accuracy: 0.9964.. Test loss: 0.5102.. Test accuracy: 0.8644.. Top-3 test accuracy: 0.9774\n",
      "Epoch 50/100.. Time per epoch: 41.1721.. Average time per step: 0.1053.. Train loss: 0.1454.. Train accuracy: 0.9482.. Top-3 train accuracy: 0.9965.. Test loss: 0.5113.. Test accuracy: 0.8609.. Top-3 test accuracy: 0.9755\n",
      "Epoch 51/100.. Time per epoch: 41.0291.. Average time per step: 0.1049.. Train loss: 0.1431.. Train accuracy: 0.9496.. Top-3 train accuracy: 0.9970.. Test loss: 0.4955.. Test accuracy: 0.8663.. Top-3 test accuracy: 0.9781\n",
      "Epoch 52/100.. Time per epoch: 41.1585.. Average time per step: 0.1053.. Train loss: 0.1354.. Train accuracy: 0.9518.. Top-3 train accuracy: 0.9973.. Test loss: 0.5051.. Test accuracy: 0.8670.. Top-3 test accuracy: 0.9751\n",
      "Epoch 53/100.. Time per epoch: 41.3139.. Average time per step: 0.1057.. Train loss: 0.1335.. Train accuracy: 0.9531.. Top-3 train accuracy: 0.9973.. Test loss: 0.5468.. Test accuracy: 0.8596.. Top-3 test accuracy: 0.9734\n",
      "Epoch 54/100.. Time per epoch: 41.0221.. Average time per step: 0.1049.. Train loss: 0.1337.. Train accuracy: 0.9530.. Top-3 train accuracy: 0.9971.. Test loss: 0.5105.. Test accuracy: 0.8693.. Top-3 test accuracy: 0.9765\n",
      "Epoch 55/100.. Time per epoch: 41.1897.. Average time per step: 0.1053.. Train loss: 0.1353.. Train accuracy: 0.9526.. Top-3 train accuracy: 0.9968.. Test loss: 0.5072.. Test accuracy: 0.8704.. Top-3 test accuracy: 0.9773\n",
      "Epoch 56/100.. Time per epoch: 41.1800.. Average time per step: 0.1053.. Train loss: 0.1298.. Train accuracy: 0.9537.. Top-3 train accuracy: 0.9973.. Test loss: 0.5095.. Test accuracy: 0.8652.. Top-3 test accuracy: 0.9769\n",
      "Epoch 57/100.. Time per epoch: 40.9052.. Average time per step: 0.1046.. Train loss: 0.1262.. Train accuracy: 0.9546.. Top-3 train accuracy: 0.9976.. Test loss: 0.5338.. Test accuracy: 0.8631.. Top-3 test accuracy: 0.9762\n",
      "Epoch 58/100.. Time per epoch: 41.1451.. Average time per step: 0.1052.. Train loss: 0.1250.. Train accuracy: 0.9557.. Top-3 train accuracy: 0.9975.. Test loss: 0.5429.. Test accuracy: 0.8625.. Top-3 test accuracy: 0.9765\n",
      "Epoch 59/100.. Time per epoch: 41.4616.. Average time per step: 0.1060.. Train loss: 0.1226.. Train accuracy: 0.9564.. Top-3 train accuracy: 0.9975.. Test loss: 0.5568.. Test accuracy: 0.8624.. Top-3 test accuracy: 0.9746\n",
      "Epoch 60/100.. Time per epoch: 41.2085.. Average time per step: 0.1054.. Train loss: 0.1222.. Train accuracy: 0.9572.. Top-3 train accuracy: 0.9977.. Test loss: 0.5183.. Test accuracy: 0.8736.. Top-3 test accuracy: 0.9775\n",
      "Epoch 61/100.. Time per epoch: 41.1972.. Average time per step: 0.1054.. Train loss: 0.1120.. Train accuracy: 0.9599.. Top-3 train accuracy: 0.9978.. Test loss: 0.5273.. Test accuracy: 0.8671.. Top-3 test accuracy: 0.9754\n",
      "Epoch 62/100.. Time per epoch: 41.2083.. Average time per step: 0.1054.. Train loss: 0.1145.. Train accuracy: 0.9602.. Top-3 train accuracy: 0.9981.. Test loss: 0.5245.. Test accuracy: 0.8676.. Top-3 test accuracy: 0.9754\n",
      "Epoch 63/100.. Time per epoch: 41.3477.. Average time per step: 0.1057.. Train loss: 0.1130.. Train accuracy: 0.9605.. Top-3 train accuracy: 0.9981.. Test loss: 0.5324.. Test accuracy: 0.8644.. Top-3 test accuracy: 0.9762\n",
      "Epoch 64/100.. Time per epoch: 41.0436.. Average time per step: 0.1050.. Train loss: 0.1123.. Train accuracy: 0.9597.. Top-3 train accuracy: 0.9980.. Test loss: 0.5284.. Test accuracy: 0.8684.. Top-3 test accuracy: 0.9756\n",
      "Epoch 65/100.. Time per epoch: 41.1151.. Average time per step: 0.1052.. Train loss: 0.1124.. Train accuracy: 0.9611.. Top-3 train accuracy: 0.9977.. Test loss: 0.5629.. Test accuracy: 0.8631.. Top-3 test accuracy: 0.9761\n",
      "Epoch 66/100.. Time per epoch: 41.3764.. Average time per step: 0.1058.. Train loss: 0.1032.. Train accuracy: 0.9637.. Top-3 train accuracy: 0.9981.. Test loss: 0.5626.. Test accuracy: 0.8698.. Top-3 test accuracy: 0.9756\n",
      "Epoch 67/100.. Time per epoch: 41.0686.. Average time per step: 0.1050.. Train loss: 0.1067.. Train accuracy: 0.9619.. Top-3 train accuracy: 0.9981.. Test loss: 0.5488.. Test accuracy: 0.8693.. Top-3 test accuracy: 0.9757\n",
      "Epoch 68/100.. Time per epoch: 41.2051.. Average time per step: 0.1054.. Train loss: 0.1028.. Train accuracy: 0.9627.. Top-3 train accuracy: 0.9981.. Test loss: 0.5174.. Test accuracy: 0.8744.. Top-3 test accuracy: 0.9771\n",
      "Epoch 69/100.. Time per epoch: 41.1094.. Average time per step: 0.1051.. Train loss: 0.1052.. Train accuracy: 0.9628.. Top-3 train accuracy: 0.9982.. Test loss: 0.5450.. Test accuracy: 0.8679.. Top-3 test accuracy: 0.9752\n",
      "Epoch 70/100.. Time per epoch: 41.2759.. Average time per step: 0.1056.. Train loss: 0.1023.. Train accuracy: 0.9644.. Top-3 train accuracy: 0.9980.. Test loss: 0.5377.. Test accuracy: 0.8697.. Top-3 test accuracy: 0.9743\n",
      "Epoch 71/100.. Time per epoch: 41.0226.. Average time per step: 0.1049.. Train loss: 0.0969.. Train accuracy: 0.9663.. Top-3 train accuracy: 0.9986.. Test loss: 0.5543.. Test accuracy: 0.8684.. Top-3 test accuracy: 0.9759\n",
      "Epoch 72/100.. Time per epoch: 41.1500.. Average time per step: 0.1052.. Train loss: 0.0970.. Train accuracy: 0.9655.. Top-3 train accuracy: 0.9986.. Test loss: 0.5731.. Test accuracy: 0.8654.. Top-3 test accuracy: 0.9762\n",
      "Epoch 73/100.. Time per epoch: 41.3113.. Average time per step: 0.1057.. Train loss: 0.0964.. Train accuracy: 0.9664.. Top-3 train accuracy: 0.9987.. Test loss: 0.5231.. Test accuracy: 0.8738.. Top-3 test accuracy: 0.9773\n",
      "Epoch 74/100.. Time per epoch: 41.2221.. Average time per step: 0.1054.. Train loss: 0.0961.. Train accuracy: 0.9669.. Top-3 train accuracy: 0.9982.. Test loss: 0.5548.. Test accuracy: 0.8703.. Top-3 test accuracy: 0.9741\n",
      "Epoch 75/100.. Time per epoch: 41.4084.. Average time per step: 0.1059.. Train loss: 0.0940.. Train accuracy: 0.9677.. Top-3 train accuracy: 0.9984.. Test loss: 0.5501.. Test accuracy: 0.8705.. Top-3 test accuracy: 0.9750\n",
      "Epoch 76/100.. Time per epoch: 41.2558.. Average time per step: 0.1055.. Train loss: 0.0931.. Train accuracy: 0.9671.. Top-3 train accuracy: 0.9985.. Test loss: 0.5513.. Test accuracy: 0.8705.. Top-3 test accuracy: 0.9755\n",
      "Epoch 77/100.. Time per epoch: 41.1326.. Average time per step: 0.1052.. Train loss: 0.0916.. Train accuracy: 0.9672.. Top-3 train accuracy: 0.9984.. Test loss: 0.5759.. Test accuracy: 0.8680.. Top-3 test accuracy: 0.9747\n",
      "Epoch 78/100.. Time per epoch: 41.0279.. Average time per step: 0.1049.. Train loss: 0.0923.. Train accuracy: 0.9677.. Top-3 train accuracy: 0.9985.. Test loss: 0.5939.. Test accuracy: 0.8651.. Top-3 test accuracy: 0.9754\n",
      "Epoch 79/100.. Time per epoch: 41.0528.. Average time per step: 0.1050.. Train loss: 0.0912.. Train accuracy: 0.9676.. Top-3 train accuracy: 0.9985.. Test loss: 0.5379.. Test accuracy: 0.8743.. Top-3 test accuracy: 0.9772\n",
      "Epoch 80/100.. Time per epoch: 41.1570.. Average time per step: 0.1053.. Train loss: 0.0888.. Train accuracy: 0.9685.. Top-3 train accuracy: 0.9986.. Test loss: 0.5514.. Test accuracy: 0.8676.. Top-3 test accuracy: 0.9773\n",
      "Epoch 81/100.. Time per epoch: 41.1286.. Average time per step: 0.1052.. Train loss: 0.0811.. Train accuracy: 0.9715.. Top-3 train accuracy: 0.9988.. Test loss: 0.5447.. Test accuracy: 0.8712.. Top-3 test accuracy: 0.9758\n",
      "Epoch 82/100.. Time per epoch: 41.1534.. Average time per step: 0.1053.. Train loss: 0.0855.. Train accuracy: 0.9695.. Top-3 train accuracy: 0.9990.. Test loss: 0.5383.. Test accuracy: 0.8723.. Top-3 test accuracy: 0.9777\n",
      "Epoch 83/100.. Time per epoch: 41.0829.. Average time per step: 0.1051.. Train loss: 0.0850.. Train accuracy: 0.9695.. Top-3 train accuracy: 0.9986.. Test loss: 0.5694.. Test accuracy: 0.8686.. Top-3 test accuracy: 0.9761\n",
      "Epoch 84/100.. Time per epoch: 41.2808.. Average time per step: 0.1056.. Train loss: 0.0836.. Train accuracy: 0.9709.. Top-3 train accuracy: 0.9987.. Test loss: 0.5795.. Test accuracy: 0.8684.. Top-3 test accuracy: 0.9768\n",
      "Epoch 85/100.. Time per epoch: 41.1202.. Average time per step: 0.1052.. Train loss: 0.0833.. Train accuracy: 0.9702.. Top-3 train accuracy: 0.9988.. Test loss: 0.5686.. Test accuracy: 0.8689.. Top-3 test accuracy: 0.9761\n",
      "Epoch 86/100.. Time per epoch: 41.3899.. Average time per step: 0.1059.. Train loss: 0.0841.. Train accuracy: 0.9699.. Top-3 train accuracy: 0.9987.. Test loss: 0.6020.. Test accuracy: 0.8642.. Top-3 test accuracy: 0.9744\n",
      "Epoch 87/100.. Time per epoch: 41.3463.. Average time per step: 0.1057.. Train loss: 0.0766.. Train accuracy: 0.9737.. Top-3 train accuracy: 0.9990.. Test loss: 0.6079.. Test accuracy: 0.8625.. Top-3 test accuracy: 0.9738\n",
      "Epoch 88/100.. Time per epoch: 41.4964.. Average time per step: 0.1061.. Train loss: 0.0817.. Train accuracy: 0.9701.. Top-3 train accuracy: 0.9989.. Test loss: 0.5869.. Test accuracy: 0.8642.. Top-3 test accuracy: 0.9760\n",
      "Epoch 89/100.. Time per epoch: 41.6869.. Average time per step: 0.1066.. Train loss: 0.0807.. Train accuracy: 0.9721.. Top-3 train accuracy: 0.9990.. Test loss: 0.5732.. Test accuracy: 0.8714.. Top-3 test accuracy: 0.9773\n",
      "Epoch 90/100.. Time per epoch: 41.5338.. Average time per step: 0.1062.. Train loss: 0.0802.. Train accuracy: 0.9717.. Top-3 train accuracy: 0.9990.. Test loss: 0.5804.. Test accuracy: 0.8672.. Top-3 test accuracy: 0.9728\n",
      "Epoch 91/100.. Time per epoch: 41.9198.. Average time per step: 0.1072.. Train loss: 0.0711.. Train accuracy: 0.9752.. Top-3 train accuracy: 0.9990.. Test loss: 0.5988.. Test accuracy: 0.8652.. Top-3 test accuracy: 0.9753\n",
      "Epoch 92/100.. Time per epoch: 41.4986.. Average time per step: 0.1061.. Train loss: 0.0729.. Train accuracy: 0.9738.. Top-3 train accuracy: 0.9989.. Test loss: 0.5666.. Test accuracy: 0.8762.. Top-3 test accuracy: 0.9749\n",
      "Epoch 93/100.. Time per epoch: 41.5746.. Average time per step: 0.1063.. Train loss: 0.0762.. Train accuracy: 0.9726.. Top-3 train accuracy: 0.9988.. Test loss: 0.5827.. Test accuracy: 0.8727.. Top-3 test accuracy: 0.9744\n",
      "Epoch 94/100.. Time per epoch: 41.2703.. Average time per step: 0.1056.. Train loss: 0.0744.. Train accuracy: 0.9734.. Top-3 train accuracy: 0.9989.. Test loss: 0.5692.. Test accuracy: 0.8733.. Top-3 test accuracy: 0.9759\n",
      "Epoch 95/100.. Time per epoch: 41.1979.. Average time per step: 0.1054.. Train loss: 0.0774.. Train accuracy: 0.9728.. Top-3 train accuracy: 0.9990.. Test loss: 0.5748.. Test accuracy: 0.8706.. Top-3 test accuracy: 0.9769\n",
      "Epoch 96/100.. Time per epoch: 41.2180.. Average time per step: 0.1054.. Train loss: 0.0731.. Train accuracy: 0.9738.. Top-3 train accuracy: 0.9988.. Test loss: 0.5561.. Test accuracy: 0.8751.. Top-3 test accuracy: 0.9764\n",
      "Epoch 97/100.. Time per epoch: 41.1480.. Average time per step: 0.1052.. Train loss: 0.0710.. Train accuracy: 0.9745.. Top-3 train accuracy: 0.9990.. Test loss: 0.5860.. Test accuracy: 0.8747.. Top-3 test accuracy: 0.9754\n",
      "Epoch 98/100.. Time per epoch: 41.3845.. Average time per step: 0.1058.. Train loss: 0.0686.. Train accuracy: 0.9758.. Top-3 train accuracy: 0.9992.. Test loss: 0.5982.. Test accuracy: 0.8719.. Top-3 test accuracy: 0.9766\n",
      "Epoch 99/100.. Time per epoch: 41.1257.. Average time per step: 0.1052.. Train loss: 0.0656.. Train accuracy: 0.9766.. Top-3 train accuracy: 0.9992.. Test loss: 0.5854.. Test accuracy: 0.8705.. Top-3 test accuracy: 0.9751\n",
      "Epoch 100/100.. Time per epoch: 41.1404.. Average time per step: 0.1052.. Train loss: 0.0697.. Train accuracy: 0.9748.. Top-3 train accuracy: 0.9991.. Test loss: 0.5897.. Test accuracy: 0.8731.. Top-3 test accuracy: 0.9774\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_ShuffleNet_Mish.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
